import scrapy


class GugongSpider(scrapy.Spider):
    name = "gugong"

    # allowed_domains = ["quotes.toscrape.com"]
    # start_urls = ["https://digicol.dpm.org.cn/cultural/queryList"]
    async def start(self):

        url = "https://digicol.dpm.org.cn/cultural/queryList"
        for i in range(1, 4796):
            yield scrapy.FormRequest(
                url,
                formdata={
                    "page": f"{i}",
                    "hasImage": "false",
                    "ancientText": "0",
                    "ranNum": "0.11754591023663818",
                },
                headers={
                    "referer": "https://www.zongheng.com/",
                    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36",
                },
                cb_kwargs={"page": i},
            )

    def parse(self, response, page):
        print(f"正在爬取{page}页")
        for item in response.json()["rows"]:
            print(item)
            yield item
